# -*- coding: utf-8 -*-
import scrapy
import re
import requests
import random
import time
from scrapy.http import Request,FormRequest
#from requests_toolbelt import MultipartEncoder

#-----导入 selenium创建浏览器对象
from selenium import webdriver
from zhihu.items import AnswerItem,ArticleItem


class ZdSpider(scrapy.Spider):
    name = 'zd'
    allowed_domains = ['zhihu.com']
    #start_urls = ['https://www.zhihu.com/topics']

    def start_requests(self):
        #return [Request('https://www.zhihu.com/signup?next=%2F',meta={'cookiejar':1},callback=self.parse)]
#-----创建火狐浏览器模拟登录----------
        browser = webdriver.Firefox()
        browser.get('https://www.zhihu.com/signup?next=%2F')

        time.sleep(5)
        browser.find_element_by_xpath('//*[@id="root"]/div/main/div/div/div/div[2]/div[2]/span').click()
        time.sleep(2)
        browser.find_element_by_xpath(
            '//*[@id="root"]/div/main/div/div/div/div[2]/div[1]/form/div[1]/div[2]/div[1]/input').clear()
        browser.find_element_by_xpath(
            '//*[@id="root"]/div/main/div/div/div/div[2]/div[1]/form/div[1]/div[2]/div[1]/input').send_keys(
            '13389176729')
        browser.find_element_by_xpath(
            '//*[@id="root"]/div/main/div/div/div/div[2]/div[1]/form/div[2]/div/div[1]/input').clear()
        browser.find_element_by_xpath(
            '//*[@id="root"]/div/main/div/div/div/div[2]/div[1]/form/div[2]/div/div[1]/input').send_keys('y124578963')
        browser.find_element_by_xpath('//*[@id="root"]/div/main/div/div/div/div[2]/div[1]/form/button').click()
        time.sleep(5)
        browser.get_screenshot_as_file(r'C:\Users\Administrator\Desktop\Python\test5.png')
#----------登陆后取出请求中的cookie------------
        list_cookie = browser.get_cookies()
        browser.quit()
#----------创建cookie字典并格式化-------------
        cookie = {}
        for item in list_cookie:
            cookie[item['name']] = item['value']
#-----------使用cokies=cookie导入cookie，并传递到下个网址--------
        return [FormRequest('https://www.zhihu.com/people/yang-17-76/activities',cookies=cookie,callback=self.parse,meta={"cookiejar":1})]
        '''
        url2 = 'https://www.zhihu.com/people/yang-17-76/activities'
        header = {
            'Connection': 'keep-alive',
            'Host': 'www.zhihu.com',
            'Referer': 'https://www.zhihu.com/',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Mobile Safari/537.36',
        }
        r = requests.get(url2, headers=header, cookies=cookie)
        data = r.text
        with open(r'zhihu-selenium-login\zhihuyang.html', 'w', encoding='utf-8') as f:
            f.write(data)
        '''

    def parse(self, response):
#------保存登陆后个人主页，并传递到话题广场-------------
        data = response.body
        with open('zhihuyang.html', 'wb') as f:
            f.write(data)
        yield Request('https://www.zhihu.com/topics',callback=self.next)

    def next(self,response):
        title_list = response.xpath('//li[@class="zm-topic-cat-item"]/a/@href').extract()
#----------取出各个父类话题的ID---------------
        title_id = response.xpath('//li[@class="zm-topic-cat-item"]/@data-id').extract()
        #print(title_list)
        for i in range(0,len(title_id)):
            offset_num = 0
            topic_id = title_id[i]
            isnext = True
#------------通过while循环取出各类父话题中所有子话题----------
            while isnext:
                #url = 'https://www.zhihu.com/topics'+str(title_id[i])
                post_url = 'https://www.zhihu.com/node/TopicsPlazzaListV2'
                header = {'User-Agent':random.choice(self.MY_USER_AGENT)}
                data = {
                    'method': 'next',
                    'params': '{"topic_id":'+topic_id+',"offset":'+str(offset_num)+',"hash_id":""}',
                }
                r = requests.post(post_url,headers=header,data=data)
                r.encoding = 'utf-8'
#--------通过POST请求取出各类子话题的ID-----------
                childid_json = r.json()
                childid_str = childid_json['msg']
                childid_pat = 'href="(.*?)"'
                childid_list = re.compile(childid_pat,re.S).findall(str(childid_str))
                offset_num += 20
                if len(childid_list) == 0:
                    isnext = False
                    continue
                for j in childid_list:
                    childid_url = 'https://www.zhihu.com'+str(j)
                    yield Request(childid_url, callback=self.next2)


    def next2(self,response):
        child_title = response.xpath('div[@class="TopicMetaCard-title"]/text()').extract()
#---------取出子话题的ID--------
        child_url = response.url
        #print(child_url)
        pat = 'https://www.zhihu.com/topic/(.*?)$'
        id = re.compile(pat).findall(child_url)[0]
        #print(id)
#--------------------------------
        curl = 'https://www.zhihu.com/api/v4/topics/' + str(id) + '/feeds/top_activity'
#---------构建post请求参数after_id---------------
        for x in range(0,100):
            after_id1 = str(random.randrange(10000))
            after_id2 = str(random.randrange(100000))
            after_id = str(after_id1+'.'+after_id2)
            header = {
                'authorization': 'oauth c3cef7c66a1843f8b3a9e6a1e3160e20',
                'User-Agent': random.choice(self.MY_USER_AGENT)
            }
            values = {'limit': '20', 'after_id': after_id}
            r = requests.get(curl,headers=header,params=values)
#-------------------------------------
            grandson_json = r.json()
            g_data_list = grandson_json['data']
            #print(len(g_data_list))
#-------------取出各子话题中的文章内容----------
            for m in range(0,len(g_data_list)):
                g_data = g_data_list[m]
                g_type = g_data['target']['type']
#-----------通过取出JSON文章类型判断是问答还是文章------------
                if (g_type == 'answer'):
                    question_id = g_data['target']['question']['id']
                    answer_id = g_data['target']['id']
                    g_url = 'https://www.zhihu.com/question/'+str(question_id)+'/answer/'+str(answer_id)
                    yield Request(g_url,callback=self.answer)
                else :
                    article_id = g_data['target']['id']
                    g_url = 'https://zhuanlan.zhihu.com/p/'+str(article_id)
                    yield Request(g_url,callback=self.article)

    def answer(self,response):
        item = AnswerItem()
        #print(response.url)
#--------取出问答的所有回答数量-------
        answer_num = int(response.xpath('//meta[@itemprop="answerCount"]/@content').extract()[0])
        question_title = response.xpath('//h1[@class="QuestionHeader-title"]/text()').extract()[0]
        question_content = response.xpath('//span[@class="RichText"]/text()').extract()[0]
#---------通过回答数量判断需要请求几次------------
        if answer_num % 20 != 0:
            pages = (answer_num // 20) + 1
        else:
            pages = answer_num / 20
        offset = 3
        #print(response.url)
#---------取出各问题的ID进行请求------
        pat = 'https://www.zhihu.com/question/(.*?)/answer'
        id = re.compile(pat).findall(response.url)[0]
        url = 'https://www.zhihu.com/api/v4/questions/'+str(id)+'/answers?'
        for page in range(0,int(pages)):
            header = {
                'Cookie': '__DAYU_PP=MmEQzRjEbv6i3AbjaBrq596e9ee000bd; q_c1=b7c03f5c08134a37aaa185fcb3428e68|1521294486000|1521294486000; _zap=b48ed69b-a140-41c7-bf1a-73477fa07bf9; capsion_ticket="2|1:0|10:1522152235|14:capsion_ticket|44:MTMyMWI5MzA2YTI1NGM1Mjk2MzczZDAxZGI5M2UzYmE=|aa59cef0e460dbcfb85a77b6b66f35b08e174d562f8d67378be79decdb0c2828"; __utma=51854390.1084335972.1521810876.1522168579.1522162626.13; __utmz=51854390.1522168579.11.7.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/topic; r_cap_id="YzJiNjM0ZTY4ZmQxNDIzMmI0ZWJmMTM0MjlmNTViODk=|1521905321|79dacac5b554804c6fb03e1c62da13a1a298dd2c"; cap_id="Zjg5NGZjNDZmNTMyNGFlNjg1Yjc0NGIzNWU2NTgwZjI=|1521905321|0432bf44a9f055458033f529af2ff0f4fb094e47"; l_cap_id="ZjVkZTMzZjM5ZmM1NDA0YTk1ZTAwM2Q2ZTRhZmEwZDM=|1521905321|6886e5448743b32e7552508c9506590dfd658737"; _xsrf=8456e42b58983e2cdaab2a3f3408b233; d_c0="AECsIywOUQ2PTsRTgH2R3qSi13yW42icgfo=|1521557832"; __utmv=51854390.100--|2=registration_date=20180318=1^3=entry_date=20180318=1; z_c0="2|1:0|10:1522152246|4:z_c0|92:Mi4xNVU1Q0NBQUFBQUFBUUt3akxBNVJEU1lBQUFCZ0FsVk5Ob0duV3dEM1RDZ0YwRmliSm9OWG9ZTmVfT3p2M2FseXZB|82941cdcce28958f0fdce95b66fb1947caee6c0897de854921b53301f65e1568"; __utmc=51854390; _xsrf=e71f8537-923a-40ef-b736-71cea4d85695; __utmb=51854390.0.10.1522162626',
                'User-Agent': random.choice(self.MY_USER_AGENT),
                'x-udid': 'AECsIywOUQ2PTsRTgH2R3qSi13yW42icgfo=',
            }
            data = {'limit':'20',
                    'offset':str(offset),
                    'sort_by':'default',
                    'include': 'data[*].is_normal,admin_closed_comment,reward_info,is_collapsed,annotation_action,annotation_detail,collapse_reason,is_sticky,collapsed_by,suggest_edit,comment_count,can_comment,content,editable_content,voteup_count,reshipment_settings,comment_permission,created_time,updated_time,review_info,relevant_info,question,excerpt,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp,upvoted_followees;data[*].mark_infos[*].url;data[*].author.follower_count,badge[?(type=best_answerer)].topics',
                    }
            r = requests.get(url,headers=header,params=data)
            answer_list = r.json()['data']
            offset += 5
#----------取出json中问题里的所有回答----------
            for l in range(0,len(answer_list)):
                answer_data = answer_list[l]
                item['question_title'] = question_title
                item['question_content'] = question_content
                item['answer_people'] = answer_data['author']['name']
                item['answer_content'] = answer_data['content']
                print(item['question_title'])
                yield item
    def article(self,response):
#-----------取出文章的信息--------------
        item = ArticleItem()
        item['title'] = response.xpath('//title[@data-react-helmet="true"]/text()').extract()
        item['people'] = response.xpath('//meta[@itemprop="name"]/@content').extract()
        item['content'] = response.xpath('//div class="RichText Post-RichText"]/p/text()').extract()
        print(item['title'])
        yield item



    MY_USER_AGENT = [
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
        "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
        "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
        "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
        "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
        "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
        "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
        "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
        "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
        "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
        "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
        "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
    ]
